var cheerio = require('cheerio')
var superagent = require('superagent')
var url = require('url')
var async = require('async')

var cnodeUrl = 'https://cnodejs.org/'

superagent.get(cnodeUrl)
    .end(function(err, res) {
        if (err) {
          return console.error(err)
        }
        var topicUrls = []
        var $ = cheerio.load(res.text)

        $('#topic_list .topic_title').each(function(idx, element){
            var $element = $(element)
            var href = url.resolve(cnodeUrl,$element.attr('href'))
            topicUrls.push(href)
        })

        var concurrencyCount = 0
        var itemArr = []
        var fetchUrl = function(topicUrl, callback){
            var delay = parseInt((Math.random() * 10000000) % 2000, 10)
            concurrencyCount ++
            console.log('并发数：'+ concurrencyCount,'正在抓取：',topicUrl,'耗时：（ms）'+delay)
            superagent.get(topicUrl)
                .end(function(err,res){
                    var $ = cheerio.load(res.text)
                    var topics = {
                        title: $('.topic_full_title').text().trim(),
                        href: topicUrl,
                        comment1: $('.reply_content').eq(0).text().trim(),
                        author1:$('.user_name a').text().trim(),
                        score1:$('.floor .big').text().trim()
                    }
                    itemArr.push(topics)
                })
            setTimeout(function(){
                concurrencyCount --
                callback(null,topicUrl + 'html content')
            },delay)
        }

        async.mapLimit(topicUrls,5,
            function(topicUrl, callback){
                fetchUrl(topicUrl, callback)
            },
            function(err, result){
                console.log('final:')
                console.log(result)
                console.log(itemArr)
            })
    })